#通过模拟浏览器操作，爬取抖音上根据指定关键词搜索到的视频的相关信息，如视频标题、发布时间、点赞数、评论数等，并将这些信息保存到一个Excel文件中。
import pymysql
from DrissionPage import ChromiumPage, ChromiumOptions  # 导入DrissionPage框架相关模块，用于浏览器操作
from DataRecorder import Recorder  # 导入数据记录模块，用于将爬取到的数据保存到文件
import time  # 导入时间模块，用于控制程序的等待时间等
import random  # 导入随机模块，用于生成随机等待时间等
import os  # 导入操作系统模块，用于文件操作等功能
import datetime  # 导入日期时间模块，用于处理时间和日期相关操作

from emotion_analysis import emotion_analysis

# 定义一个列表，用于存储所有爬取到的视频id
all_videos_id = []
#
# 定义要搜索的关键词
keyword = '遇龙河'

''' 连接数据库
主机：jzhangluo.com
端口号：1706
用户：root
密码：guet1314999hire
'''

cnx = pymysql.connect(
    host="jzhangluo.com",
    port=1706,
    user="root",
    password="guet1314999hire",
    database="monitor"
 )

# # 创建一个游标对象
cursor = cnx.cursor()

# 定义一个函数，用于删除初始化的excel文件
def delete_file(file_path):
    """用于删除初始化excel文件"""
    # 检查文件是否存在
    if os.path.exists(file_path):
        # 删除文件
        os.remove(file_path)
        print(f"已删除初始化excel文件：{file_path}")
    else:
        print(f"文件不存在：{file_path} ")

# 定义主函数
def run():
    # 设置浏览器可执行文件路径
    path = r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'
    # 设置浏览器选项，指定浏览器路径等
    co = ChromiumOptions().set_browser_path(path)
    co.auto_port(True)

    # 创建浏览器页面对象
    page = ChromiumPage(addr_or_opts=co)
    # 等待2秒，让浏览器加载完成
    time.sleep(2)
    # 构造搜索关键词的抖音链接
    url = f'https://www.douyin.com/search/{keyword}?type=video'

    # 这里是抖音登录后的cookie值，用于模拟登录状态，需要根据实际情况更新
    cookies = 'SEARCH_RESULT_LIST_TYPE=%22single%22; hevc_supported=true; fpk1=U2FsdGVkX1+dFbZOQEOEL6rSEvn2w2hWAa4Z7EBGWMCNjY5CjkmQlow1U126/eFbTZ+6syMyw1dOR3V+w/IYeQ==; fpk2=8369da3c75ccd12bc017791df73a85c8; passport_csrf_token=0ae7f8870879d9068ca9ba510a46b4cc; passport_csrf_token_default=0ae7f8870879d9068ca9ba510a46b4cc; __security_mc_1_s_sdk_crypt_sdk=6d1222a3-4a6b-9a16; bd_ticket_guard_client_web_domain=2; enter_pc_once=1; UIFID=c8c20d54553eadab8c678961c2b0df95555df87bbc6b890988ad105aec15abc0fda41018c4912e42d41b29315b1da0934c5534c700697702f8f35698748221d30772e402dbdbfc9f308f22dce448059cd446d88c068bb056f6516d6ed017674d79079c92818c0f272089194744aa26f1e6d25f03b8ccd837f1759d67f10cdbfe713dd0c1fee43adf96261105fa030ec751ce8ad34bcd5c98de2c2cd9f1a4d308faea6a50101260d81a01d81c96e4351a08489833be8e2760168c48f37df94f93; s_v_web_id=verify_mbut9tfj_GvATKmol_Gpl5_40Q1_BTpO_lWV2zljyTWKT; dy_swidth=1920; dy_sheight=1080; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1920%2C%5C%22screen_height%5C%22%3A1080%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A8%2C%5C%22device_memory%5C%22%3A8%2C%5C%22downlink%5C%22%3A10%2C%5C%22effective_type%5C%22%3A%5C%224g%5C%22%2C%5C%22round_trip_time%5C%22%3A200%7D%22; is_dash_user=1; xgplayer_user_id=256433950052; __security_mc_1_s_sdk_sign_data_key_sso=f63f6a57-4274-b57b; WallpaperGuide=%7B%22showTime%22%3A1750170847187%2C%22closeTime%22%3A0%2C%22showCount%22%3A1%2C%22cursor1%22%3A10%2C%22cursor2%22%3A2%7D; passport_mfa_token=CjYxAsaKDCrZykyYWF0mGiMm0NvOW7sir4r%2BrYW%2FlEBLGDnHkAr0WvxnG5WHwsnv1AOvxxS3k8EaSgo8AAAAAAAAAAAAAE8gCtsLypVE2RXho4JW1mDUSVycTLAxb7GEcLQrydD79HWRzBYJiD1BOQEM5uoXkbT5EKWs9A0Y9rHRbCACIgEDZuLviA%3D%3D; d_ticket=d40cc227746f44ed889817278a914d136af2c; passport_assist_user=CkDKWFsEOEalthbtaldBz-yPAgT60cKuBlhg2CXIsB0o3mhTzcn49U_j7JPbMSa41iyTEqKGxe6FzKprX_yUx8SBGkoKPAAAAAAAAAAAAABPIIRXF47UBnrezRFB9GFRDiWAUMuCb60jfUG8_9MYnGNkPqNM-jXw-wmEFIMg-soDMxDqrvQNGImv1lQgASIBA1Vgx1w%3D; n_mh=6AnakESG4iqnTu_AZeiCeQ1SI9IczC8oNSI1gDMkj3M; sid_guard=110f2037e14ab515228d1f9c69c1e829%7C1750170936%7C5184000%7CSat%2C+16-Aug-2025+14%3A35%3A36+GMT; uid_tt=d951230942d398c4dbb59a291cd90225; uid_tt_ss=d951230942d398c4dbb59a291cd90225; sid_tt=110f2037e14ab515228d1f9c69c1e829; sessionid=110f2037e14ab515228d1f9c69c1e829; sessionid_ss=110f2037e14ab515228d1f9c69c1e829; is_staff_user=false; sid_ucp_v1=1.0.0-KDQ2OGYxMzk2MzU1Y2FiNDI2ODAyNjdhYjQzNWZiODViNDhiYjYyNGUKIAiwptDPkcxOELj6xcIGGO8xIAwwh8DGqAY4B0D0B0gEGgJsZiIgMTEwZjIwMzdlMTRhYjUxNTIyOGQxZjljNjljMWU4Mjk; ssid_ucp_v1=1.0.0-KDQ2OGYxMzk2MzU1Y2FiNDI2ODAyNjdhYjQzNWZiODViNDhiYjYyNGUKIAiwptDPkcxOELj6xcIGGO8xIAwwh8DGqAY4B0D0B0gEGgJsZiIgMTEwZjIwMzdlMTRhYjUxNTIyOGQxZjljNjljMWU4Mjk; login_time=1750170936028; SelfTabRedDotControl=%5B%5D; FOLLOW_LIVE_POINT_INFO=%22MS4wLjABAAAA-RMv7pzJxV6cgBe-ybFUNZjl7BjJgALal707DxtTXeI%2F1750176000000%2F0%2F1750170939639%2F0%22; _bd_ticket_crypt_cookie=e3cfc674bdcad0b266af3b2e9a88d129; __security_mc_1_s_sdk_sign_data_key_web_protect=11cede4a-4691-8978; __security_mc_1_s_sdk_cert_key=db5c0c05-4e19-826a; __security_server_data_status=1; publish_badge_show_info=%220%2C0%2C0%2C1750170949041%22; download_guide=%223%2F20250617%2F1%22; __ac_nonce=0685192b800d99148692e; __ac_signature=_02B4Z6wo00f013y6sXQAAIDCo-FCOVtg7.N8mrXAALeK81; douyin.com; xg_device_score=7.554258647749727; device_web_cpu_core=8; device_web_memory_size=8; architecture=amd64; IsDouyinActive=true; strategyABtestKey=%221750176442.2%22; FOLLOW_NUMBER_YELLOW_POINT_INFO=%22MS4wLjABAAAA-RMv7pzJxV6cgBe-ybFUNZjl7BjJgALal707DxtTXeI%2F1750262400000%2F0%2F1750176442410%2F0%22; stream_player_status_params=%22%7B%5C%22is_auto_play%5C%22%3A0%2C%5C%22is_full_screen%5C%22%3A0%2C%5C%22is_full_webscreen%5C%22%3A0%2C%5C%22is_mute%5C%22%3A1%2C%5C%22is_speed%5C%22%3A1%2C%5C%22is_visible%5C%22%3A1%7D%22; volume_info=%7B%22volume%22%3A0.6%2C%22isMute%22%3Atrue%7D; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCRndZekFMTjRaRDZmRXNpM0NQUnpRRXNadmtzUnNoTVVVWWtVbnE4SEIwZkszcm9IZ2djYjRRbzlaRjJiSXNwRVlkaGNLTzZiVnE3QW5aNUJJbUcyQ3c9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; ttwid=1%7CIIsgKIuOhL3h67xu1zSgLsXvEX2K0-WHHCwD_aY8Hc0%7C1750176444%7Caf1656bd56e226c45c4aabeccae81ba4ba8e175ea7561d27c45ae599f37144bb; biz_trace_id=34ed78ac; odin_tt=c0247c5d1556050edd28c57c0ae6fd38b4db27b9442006b4f22b9a1cf901a57f1e27a1b26b9f89519e38dfba6328737f0b4780efc603594aafd07b33c20e1377; home_can_add_dy_2_desktop=%221%22'    # 将cookie字符串转换为字典列表，方便后续设置cookie
    cookies_list = []
    for item in cookies.split('; '):
        # 跳过空字符串
        if not item.strip():
            continue

        parts = item.split('=', 1)  # 只分割一次，避免值中包含等号的情况

        # 确保至少有一个键和一个值
        if len(parts) >= 2:
            name = parts[0].strip()
            value = parts[1].strip()
            cookies_list.append({'name': name, 'value': value})
        else:
            print(f"无效的Cookie项: {item}")

    # 打印处理后的Cookie数量，用于调试
    print(f"成功解析 {len(cookies_list)} 个Cookie")

    # 开始监听抖音的某个接口，用于获取动态加载的数据
    page.listen.start('www.douyin.com/aweme/v1/web/search/item', method='GET')
    time.sleep(2)
    # 打开抖音搜索链接
    page.get(url)
    time.sleep(2)
    # 添加cookies，模拟登录状态
    page.set.cookies(cookies)
    page.wait(3)
    # 刷新页面，使得cookie生效
    page.refresh()
    page.wait(3)
    # 获取当前时间，格式化当前时间，填充到excel表名称后缀
    current_time = time.localtime()
    formatted_time = time.strftime("%Y-%m-%d %H%M%S", current_time)
    # # 新建一个excel表格，用来保存数据
    # init_file_path = f'抖音搜索-{keyword}-{formatted_time}.xlsx'
    # r = Recorder(path=init_file_path, cache_size=500)

    # 循环提取视频
    while True:
        # 生成一个随机时间
        random_time = random.uniform(2, 3)
        # 下滑页面，随机等待，模拟人工浏览
        page.scroll.to_bottom()
        time.sleep(random_time)
        # 再次下滑页面，随机等待
        page.scroll.to_bottom()
        time.sleep(random_time)

        # 等待并获取一个数据包
        res = page.listen.wait(timeout=5)
        try:
            # 获取数据包中的json数据
            json_data = res.response.body
        except:
            print('没有监听到更多视频，结束运行。')
            break
        # 获取所有视频列表
        videos = json_data['data']

        # 提取视频信息
        for v in videos:
            # 提取视频id
            aweme_id = v['aweme_info']['aweme_id']
            # 提取视频标题
            title = v['aweme_info']['desc'].strip()
            # 提取发布时间
            create_time = v['aweme_info']['create_time']
            # 提取作者名称
            nickname = v['aweme_info']['author']['nickname']
            # 提取作者id
            uid = v['aweme_info']['author']['uid']
            # 提取认证信息
            enterprise_verify_reason = v['aweme_info']['author']['enterprise_verify_reason']
            # 提取作者主页地址
            sec_uid = v['aweme_info']['author']['sec_uid']
            # 提取作者粉丝数
            follower_count = v['aweme_info']['author']['follower_count']
            # 提取点赞数
            digg_count = v['aweme_info']['statistics']['digg_count']
            # 提取评论数
            comment_count = v['aweme_info']['statistics']['comment_count']
            # 提取收藏数
            collect_count = v['aweme_info']['statistics']['collect_count']
            # 提取转发数
            share_count = v['aweme_info']['statistics']['share_count']

            # 构造视频链接
            video_url = "https://www.douyin.com/video/" + aweme_id

            # 构造作者主页地址
            sec_uid = "https://www.douyin.com/user/" + sec_uid

            # 将时间戳转换为datetime对象
            create_time = datetime.datetime.fromtimestamp(create_time)
            # 将datetime对象转换为字符串
            create_time = create_time.strftime('%Y-%m-%d %H:%M:%S')

            # 将视频id添加到列表中
            all_videos_id.append(aweme_id)
            # 打印视频的一些关键信息，方便查看
            print(aweme_id, title, digg_count, comment_count, collect_count, share_count)
            # 构造一个字典，存储视频的所有信息
            info = {'关键词': keyword, '视频标题': title, '发布时间': create_time, '点赞': digg_count,
                    '评论': comment_count, '收藏': collect_count, '转发': share_count, '作者名称': nickname,
                    '作者粉丝数': follower_count, '视频ID': aweme_id, '视频链接': video_url, '作者主页地址': sec_uid,
                    '作者ID': uid, '作者认证信息': enterprise_verify_reason}
            # 将视频信息添加到数据记录器中
            # 判断情绪

            emotion = emotion_analysis(title)
            time.sleep(0.5)
            # r.add_data(info)
            # 执行插入数据的SQL语句
            sql = "INSERT INTO article (authorName, articleTitle, content, type, ip,spread,time, originalLink, platformID) VALUES ( %s, %s, %s, %s,%s,%s,%s,%s,20)"
            values = (nickname, title, title,  emotion, sec_uid, digg_count,create_time, video_url)
            cursor.execute(sql, values)

            # 提交事务
            cnx.commit()
        # 打印当前总共获取到的视频数量
        print(f'总计获取到{len(all_videos_id)}条视频')
        # 判断是否还有更多视频，如果有，继续循环；如果没有，结束循环
        has_more = json_data['has_more']
        if has_more == 1:
            continue
        else:
            break
    # # 爬虫结束，强制保存文件，避免数据丢失
    # r.record()

    # print('**' * 10)

    # # 打印总共获取到的视频数量和关键词
    # print(
    #     f'抖音关键词【{keyword}】总计获取到【{len(all_videos_id)}】条视频')
    # # 构造最终的文件名
    # final_file_path = f'抖音搜索-{keyword}-{len(all_videos_id)}条视频-{formatted_time}.xlsx'
    # # 将文件另存为最终文件名
    # r.record(final_file_path)
    # print(f'成功将文件另存为：{final_file_path}')

    # 删除初始文件
    # delete_file(init_file_path)


# 程序的入口
if __name__ == '__main__':
    run()
    # # 关闭游标和数据库连接
    cnx.close()
    cursor.close()

